import scrapy
from testproject.items import TestprojectItem
from scrapy_redis.spiders import RedisSpider
import redis
count = 1

class BilbilSpider(RedisSpider):
    name = "cque"
    # # allowed_domains = ["www.bilibili.com"]
    # start_urls = ["https://www.cque.edu.cn/index/zhyw.htm"]

    redis_key = "test"
    def __init__(self, *args, **kwargs):
        domain = kwargs.pop("domain", "")
        self.allowed_domains = filter(None, domain.split(","))
        super().__init__(*args, **kwargs)

    def url_list(self):
        r = redis.Redis(host="127.0.0.1",port=6379)
        urls=[]
        for i in range(1,93):
            url = "https://www.cque.edu.cn/index/zhyw/{}.htm".format(i)
            urls.append(url)
        r.lpush("test",urls)
        return urls

    def parse(self, response):
        # self.url_list()
        vider_list = response.xpath("//div[@class='faculty-all']/div/div/a")
        print(len(vider_list))
        global count
        count += 1
        for vider in vider_list:
            title = vider.xpath(".//div/text()").extract_first()
            print(title)
            url = vider.xpath('./@href').extract_first()
            item = TestprojectItem(title=title,url=url)
            yield item
        # urllist = self.url_list()
        # for url_back in urllist:
        #     yield scrapy.Request(url_back,callback=self.parse)
